package contentSpider;

import java.io.IOException;
import java.util.Iterator;
import java.util.List;
import org.jdom.Document;
import org.jdom.Element;
import org.jdom.JDOMException;
import org.jdom.input.SAXBuilder;
import util.kwdDigg.SpiderRobotKWD;
import util.urlSpider.SpiderRobotURL;

public class EngSpider {

	public static void main(String[] args) {
		// 读取xml
		// 定义从xml文件中读出的参数
		String dBName = null;
		String dBHost = null;
		// 开始读取xml文件

		System.setProperty("LOG_DIR", "./");

		// Log logger = LogFactory.getLog(EngSpider.class);

		SAXBuilder builder = new SAXBuilder(false);
		List<?> commondParameterList = null;
		List<?> kwdParameterList = null;
		List<?> sourceSiteParameterList = null;
		try {
			Document doc = builder.build("spiderSuit.xml");
			Element parameter = doc.getRootElement();
			commondParameterList = parameter.getChildren("commond");
			kwdParameterList = parameter.getChildren("kwd");
			sourceSiteParameterList = parameter.getChildren("sourceSite");
		} catch (JDOMException e) {
			e.printStackTrace();
		} catch (IOException e) {
			e.printStackTrace();
		}
		// 一次循环拿到一个采集目标网站的配置也就是xml文件中的一个sourceSite
		for (Iterator<?> iter = commondParameterList.iterator(); iter.hasNext();) {
			Element element = (Element) iter.next();
			dBName = element.getChildTextTrim("dBName");
			dBHost = element.getAttributeValue("host");
			element = null;
		}
		for (Iterator<?> iter = kwdParameterList.iterator(); iter.hasNext();) {
			Element element = (Element) iter.next();
			String first = element.getChildTextTrim("first");
			String end = element.getChildTextTrim("end");
			String urlPattern = element.getChildTextTrim("urlPattern");
			String area = element.getChildTextTrim("area");
			String proxyString = element.getChildTextTrim("proxy");
			String cookiesString = element.getChildTextTrim("cookies");
			SpiderRobotKWD srk = new SpiderRobotKWD();
			srk.setArea(area);
			srk.setCookiesString(cookiesString);
			srk.setdBName(dBName);
			srk.setEnd(end);
			srk.setFirst(first);
			srk.setHost(dBHost);
			srk.setProxyString(proxyString);
			srk.setUrlPattern(urlPattern);
			boolean sw = true;
			while (sw)
				try {
					srk.run();
					sw = false;
				} catch (Exception e) {
					continue;
					// e.printStackTrace();
				}

			srk = null;
			element = null;
			first = null;
			end = null;
			urlPattern = null;
			area = null;
			proxyString = null;
			cookiesString = null;
		}
		for (Iterator<?> iter = sourceSiteParameterList.iterator(); iter.hasNext();) {
			Element element = (Element) iter.next();
			String end = element.getChildTextTrim("end");
			String first = element.getChildTextTrim("first");
			String head = element.getChildTextTrim("head");
			String urlPattern = element.getChildTextTrim("urlPattern");
			String codeNo = element.getChildTextTrim("codeNo");
			String pagingRex = element.getChildTextTrim("pagingRex");
			String pagingRexHead = element.getChildTextTrim("pagingRexHead");
			String lookRex = element.getChildTextTrim("lookRex");
			int sourceNet = new Integer(element.getChildTextTrim("sourceNet")).intValue();
			String proxyURL = element.getChildTextTrim("proxyURL");
			String cookiesURL = element.getChildTextTrim("cookiesURL");
			String titleRex = element.getChildTextTrim("titleRex");
			String contentget = element.getChildTextTrim("contentget");
			String contentdel = element.getChildTextTrim("contentdel");
			String proxyCon = element.getChildTextTrim("proxyCon");
			String cookiesCon = element.getChildTextTrim("cookiesCon");
			SpiderRobotURL srl = new SpiderRobotURL();
			srl.setCodeNo(codeNo);
			srl.setCookiesString(cookiesURL);
			srl.setdBName(dBName);
			srl.setEnd(end);
			srl.setFirst(first);
			srl.setHead(head);
			srl.setHost(dBHost);
			srl.setLookRex(lookRex);
			srl.setProxyString(proxyURL);
			srl.setSourceNet(sourceNet);
			// System.out.println(sourceNet);
			srl.setPagingRex(pagingRex);
			srl.setPagingRexHead(pagingRexHead);
			srl.setUrlPattern(urlPattern);		
			boolean sw2 = true;
			while (sw2)
				try {
					srl.run();
					sw2 = false;
				} catch (Exception e) {
					continue;
					// e.printStackTrace();
				}
			
			ContentSpider cs = new ContentSpider();
			cs.setContentRex(contentget);
			cs.setCookiesString(cookiesCon);
			cs.setdBHost(dBHost);
			cs.setdBName(dBName);
			cs.setDelRex(contentdel);
			cs.setProxyString(proxyCon);
			cs.setSourceNet(sourceNet);
			cs.setTitleRex(titleRex);
			boolean sw = true;
			while (sw)
				try {
					cs.contentSpider();
					sw = false;
				} catch (Exception e) {
					 e.printStackTrace();
					continue;
					//
				}
			cs = null;
			srl = null;
			element = null;
			end = null;
			first = null;
			head = null;
			urlPattern = null;
			codeNo = null;
			pagingRex = null;
			pagingRexHead = null;
			lookRex = null;
			proxyURL = null;
			cookiesURL = null;
			titleRex = null;
			contentget = null;
			contentdel = null;
			proxyCon = null;
			cookiesCon = null;

		}
		
		dBName = null;
		dBHost = null;

	}
}
